import os
import re

from util import Index, InvertedIndex


def split_words(filename, file, index):
    lz = file.readlines()
    i = -1
    for line in lz:
        word_list = line.split()
        for word in word_list:
            while len(word) > 0 and word[-1] in ['.', ',', ':', '-']:
                word = word[:-1]
            word = re.sub(r'[^\w\s\d-]', '', word)
            i += 1
            if len(word) == 0:
                continue
            index.addKeyWord(word.lower(), filename, i)


def build_index(file_list) -> (Index, InvertedIndex):
    index = Index()
    for filename, file in file_list:
        split_words(filename, file, index)
    return index, InvertedIndex().toInverted(index)


def get_file_list() -> list:
    ret = []
    path = 'file'
    files = os.listdir(path)#返回文件名的列表
    for filename in files:
        if filename.endswith("txt"):#后缀为txt
            f = open(path + os.sep + filename, 'r', encoding='utf8')
            ret.append((filename, f))
            print(f.name)
    return ret


def gc(file_list):
    for filename, file in file_list:
        file.close()


if __name__ == "__main__":
    a, b = build_index(get_file_list())
    print(a.map)
    print(b.map)
